﻿using System;
using System.Net;
using System.Text;
using System.Xml;
using Microsoft.Practices.ServiceLocation;
using OMR.Crawler.Core;
using OMR.Crawler.Core.Collections;
using OMR.Crawler.Core.Crawlers;
using SolrNet;

namespace OMR.Crawler.SOLRExample
{
    class Program
    {
        private static string __ENTRYPOINT = "d:\\";

        private static DateTime started;
        private static int fileCount = 0;
        static ISolrOperations<CrawlDocument> solr;

        static void Main(string[] args)
        {
            Startup.Init<CrawlDocument>("http://localhost:8983/solr");
            solr = ServiceLocator.Current.GetInstance<ISolrOperations<CrawlDocument>>();

            //solr.Delete(SolrQuery.All);
            solr.Commit();

            var settings = new MyCollection();
            settings.SetItem(new MyCollectionItem() { Name = "EntryPoint", Value = __ENTRYPOINT });
            settings.SetItem(new MyCollectionItem() { Name = "MaxThreadCount", Value = 200 });

            var cr = CrawlerFactory.Create(settings);

            cr.OnCrawlStarted += cr_OnCrawlStarted;
            cr.OnItemCrawled += cr_OnItemCrawled;
            cr.OnCawlError += cr_OnCawlError;
            cr.OnCrawlFinished += cr_OnCrawlFinished;
            cr.Begin();

            Console.WriteLine("*iki tuşa basın");
            Console.ReadKey();
        }

        static void cr_OnCrawlFinished(object sender, CrawlFinishedEventArgs e)
        {
            lock (_lockObj)
            {
                solr.Commit();

                Console.ForegroundColor = ConsoleColor.Green;
                Console.WriteLine((DateTime.Now - started).ToString());
                Console.WriteLine(fileCount);
                Console.ResetColor();
            }
        }

        static void cr_OnCrawlStarted(object sender, CrawlStartedEventArgs e)
        {
            lock (_lockObj)
            {
                started = DateTime.Now;
            }
        }

        static void cr_OnCawlError(object sender, CrawlErrorEventArgs e)
        {
            lock (_lockObj)
            {
                Console.ForegroundColor = ConsoleColor.Red;
                Console.WriteLine(e.Exception.Message);
                Console.ResetColor();
            }
        }

        static void cr_OnItemCrawled(object sender, CrawledItemEventArgs e)
        {
            System.Threading.Interlocked.Increment(ref fileCount);

            lock (_lockObj)
            {
                SendToSolr(e.Current);
                //var t = new Thread(new ParameterizedThreadStart(SendToSolr));
                //t.Start(e.Current);

                System.Console.WriteLine(e.Current.DeepCount + ":[" + e.Current.Content.Length + "] " + e.Current.EntryPoint);
            }
        }

        private static MultiThreadQueue<CrawlWork> solrIndexingQueue = new MultiThreadQueue<CrawlWork>(100);

        static void SendToSolr(object obj)
        {
            var cw = (CrawlWork)obj;

            try
            {
                solrIndexingQueue.Enqueue(cw);

                if (fileCount % 1000 == 0)
                {
                    Console.WriteLine("commit:" + fileCount);
                    solr.Commit();
                }

                solrIndexingQueue.ProceedMultiThread((x) =>
                {
                    try
                    {
                        WebClient wc = new WebClient();
                        var s = wc.UploadData("http://localhost:8983/solr/update/extract?&wt=xml&extractOnly=true&indent=true&extractFormat=text", x.Content);
                        var d = Encoding.UTF8.GetString(s);

                        XmlDocument sd = new XmlDocument();
                        sd.LoadXml(d);

                        CrawlDocument cd = new CrawlDocument();
                        cd.Content = sd.SelectSingleNode("/response/str").InnerText;
                        cd.Location = x.EntryPoint;
                        cd.Size = x.Content.Length;

                        cd.ContentType = x.MimeType;
                        if (cd.ContentType == null)
                        {
                            var asd = sd.SelectSingleNode("response/lst/arr[@name='Content-Type']/str");
                            if (asd != null)
                            {
                                cd.ContentType = asd.InnerText.Split(';')[0];
                            }
                        }

                        cd.Title = x.Title;
                        if (cd.Title == null)
                        {
                            var asd = sd.SelectSingleNode("response/lst/arr[@name='title']/str");
                            if (asd != null)
                            {
                                cd.Title = asd.InnerText;
                            }
                            //if (sd.SelectSingleNode)
                            //{

                            //}
                        }

                        if (cd.Title == null)
                        {
                            cd.Title = string.Empty;
                        }

                        solr.Add(cd);

                    }
                    catch (Exception exception)
                    {
                        return;
                    }
                    //solr.Commit();
                });
            }
            catch (Exception exception)
            {
                cr_OnCawlError(null, new CrawlErrorEventArgs() { Exception = exception });
            }
        }

        private static object _lockObj = new object();

    }
}
